{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip uninstall --yes h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEPENDENCIES ########################################################################################################################################\n",
    "\n",
    "!pip install keras\n",
    "!pip install keras_vggface\n",
    "!pip install pandas\n",
    "!pip install scikit_image\n",
    "!pip install h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORTS #############################################################################################################################################\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow.python.lib.io import file_io\n",
    "from skimage.transform import resize\n",
    "from keras import backend as K\n",
    "from keras.utils import to_categorical\n",
    "from keras_vggface.vggface import VGGFace\n",
    "from keras.models import Model\n",
    "from keras.layers import Flatten, Dense \n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import TensorBoard, LearningRateScheduler, ReduceLROnPlateau, EarlyStopping, Callback\n",
    "import h5py # For saving the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PARAMETERS ##########################################################################################################################################\n",
    "\n",
    "# Folder where logs and models are stored\n",
    "folder = 'logs/ResNet-50'\n",
    "\n",
    "# Size of the images\n",
    "img_height, img_width = 197, 197\n",
    "\n",
    "# Parameters\n",
    "num_classes         = 7     # ['Anger', 'Disgust', 'Fear', 'Happiness', 'Sadness', 'Surprise', 'Neutral']\n",
    "epochs_top_layers   = 5\n",
    "epochs_all_layers   = 100\n",
    "batch_size          = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATASETS ############################################################################################################################################\n",
    "\n",
    "# Folder where logs and models are stored\n",
    "folder = 'gs://emotion_recognition/logs/ResNet-50'\n",
    "\n",
    "# Data paths\n",
    "train_dataset\t= 'gs://emotion_recognition/FER-2013/fer2013_train.csv'\n",
    "eval_dataset \t= 'gs://emotion_recognition/FER-2013/fer2013_eval.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MODEL ###############################################################################################################################################\n",
    "\n",
    "# Create the based on ResNet-50 architecture pre-trained model\n",
    "    # model:        Selects one of the available architectures vgg16, resnet50 or senet50\n",
    "    # include_top:  Whether to include the fully-connected layer at the top of the network\n",
    "    # weights:      Pre-training on VGGFace\n",
    "    # input_shape:  Optional shape tuple, only to be specified if include_top is False (otherwise the input\n",
    "    #               shape has to be (224, 224, 3) (with 'channels_last' data format) or (3, 224, 224) (with\n",
    "    #               'channels_first' data format). It should have exactly 3 inputs channels, and width and\n",
    "    #               height should be no smaller than 197. E.g. (200, 200, 3) would be one valid value.\n",
    "# Returns a keras Model instance\n",
    "base_model = VGGFace(\n",
    "    model       = 'resnet50',\n",
    "    include_top = False,\n",
    "    weights     = 'vggface',\n",
    "    input_shape = (img_height, img_width, 3))\n",
    "\n",
    "# Places x as the output of the pre-trained model\n",
    "x = base_model.output\n",
    "\n",
    "# Flattens the input. Does not affect the batch size\n",
    "x = Flatten()(x)\n",
    "\n",
    "# Add a fully-connected layer and a logistic layer\n",
    "# Dense implements the operation: output = activation(dot(input, kernel) + bias(only applicable if use_bias is True))\n",
    "    # units:        Positive integer, dimensionality of the output space\n",
    "    # activation:   Activation function to use\n",
    "    # input shape:  nD tensor with shape: (batch_size, ..., input_dim)\n",
    "    # output shape: nD tensor with shape: (batch_size, ..., units)\n",
    "x = Dense(1024, activation = 'relu')(x)\n",
    "predictions = Dense(num_classes, activation = 'softmax')(x)\n",
    "\n",
    "# The model we will train\n",
    "model = Model(inputs = base_model.input, outputs = predictions)\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATA PREPARATION ####################################################################################################################################\n",
    "\n",
    "# Preprocesses a numpy array encoding a batch of images\n",
    "    # x: Input array to preprocess\n",
    "def preprocess_input(x):\n",
    "    x -= 128.8006 # np.mean(train_dataset)\n",
    "    return x\n",
    "\n",
    "# Function that reads the data from the csv file, increases the size of the images and returns the images and their labels\n",
    "    # dataset: Data path\n",
    "def get_data(dataset):\n",
    "    file_stream = file_io.FileIO(dataset, mode='r')\n",
    "    data = pd.read_csv(file_stream)\n",
    "    pixels = data['pixels'].tolist()\n",
    "    images = np.empty((len(data), img_height, img_width, 3))\n",
    "    i = 0\n",
    "\n",
    "    for pixel_sequence in pixels:\n",
    "        single_image = [float(pixel) for pixel in pixel_sequence.split(' ')]  # Extraction of each single\n",
    "        single_image = np.asarray(single_image).reshape(48, 48) # Dimension: 48x48\n",
    "        single_image = resize(single_image, (img_height, img_width), order = 3, mode = 'constant') # Dimension: 139x139x3 (Bicubic)\n",
    "        ret = np.empty((img_height, img_width, 3))  \n",
    "        ret[:, :, 0] = single_image\n",
    "        ret[:, :, 1] = single_image\n",
    "        ret[:, :, 2] = single_image\n",
    "        images[i, :, :, :] = ret\n",
    "        i += 1\n",
    "    \n",
    "    images = preprocess_input(images)\n",
    "    labels = to_categorical(data['emotion'])\n",
    "\n",
    "    return images, labels    \n",
    "\n",
    "# Data preparation\n",
    "train_data_x, train_data_y  = get_data(train_dataset)\n",
    "val_data  = get_data(eval_dataset)\n",
    "\n",
    "# Generate batches of tensor image data with real-time data augmentation. The data will be looped over (in batches) indefinitely\n",
    "# rescale:          Rescaling factor (defaults to None). Multiply the data by the value provided (before applying any other transformation)\n",
    "# rotation_range:   Int. Degree range for random rotations\n",
    "# shear_range:      Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)\n",
    "# zoom_range:       Float or [lower, upper]. Range for random zoom. If a float, [lower, upper] = [1-zoom_range, 1+zoom_range]\n",
    "# fill_mode :       Points outside the boundaries of the input are filled according to the given mode: {\"constant\", \"nearest\", \"reflect\" or \"wrap\"}\n",
    "# horizontal_flip:  Boolean. Randomly flip inputs horizontally\n",
    "train_datagen = ImageDataGenerator(\n",
    "    rotation_range  = 10,\n",
    "    shear_range     = 10, # 10 degrees\n",
    "    zoom_range      = 0.1,\n",
    "    fill_mode       = 'reflect',\n",
    "    horizontal_flip = True)\n",
    "\n",
    "# Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batcfillhes indefinitely, in an infinite loop\n",
    "    # x:            Data. Should have rank 4. In case of grayscale data, the channels axis should have value 1, and in case of RGB data, \n",
    "    #               it should have value 3\n",
    "    # y:            Labels\n",
    "    # batch_size:   Int (default: 32)\n",
    "train_generator = train_datagen.flow(\n",
    "    train_data_x,\n",
    "    train_data_y,\n",
    "    batch_size  = batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# UPPER LAYERS TRAINING ###############################################################################################################################\n",
    "\n",
    "# First: train only the top layers (which were randomly initialized) freezing all convolutional ResNet-50 layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# Compile (configures the model for training) the model (should be done *AFTER* setting layers to non-trainable)\n",
    "    # optimizer:    String (name of optimizer) or optimizer object\n",
    "        # lr:       Float >= 0. Learning rate\n",
    "        # beta_1:   Float, 0 < beta < 1. Generally close to 1\n",
    "        # beta_2:   Float, 0 < beta < 1. Generally close to 1\n",
    "        # epsilon:  Float >= 0. Fuzz factor\n",
    "        # decay:    Float >= 0. Learning rate decay over each update\n",
    "    # loss:     String (name of objective function) or objective function\n",
    "    # metrics:  List of metrics to be evaluated by the model during training and testing\n",
    "model.compile(\n",
    "    optimizer   = Adam(lr = 1e-3, beta_1 = 0.9, beta_2 = 0.999, epsilon = 1e-08, decay = 0.0), \n",
    "    loss        = 'categorical_crossentropy', \n",
    "    metrics     = ['accuracy'])\n",
    "\n",
    "# This callback writes a log for TensorBoard, which allows you to visualize dynamic graphs of your training and test metrics, \n",
    "# as well as activation histograms for the different layers in your model\n",
    "    # log_dir:          The path of the directory where to save the log files to be parsed by TensorBoard\n",
    "    # histogram_freq:   Frequency (in epochs) at which to compute activation and weight histograms for the layers of the model\n",
    "    #                   If set to 0, histograms won't be computed. Validation data (or split) must be specified for histogram visualizations\n",
    "    # write_graph:      Whether to visualize the graph in TensorBoard. The log file can become quite large when write_graph is set to True\n",
    "    # write_grads:      Whether to visualize gradient histograms in TensorBoard. histogram_freq must be greater than 0\n",
    "    # write_images:     Whether to write model weights to visualize as image in TensorBoard\n",
    "# To visualize the files created during training, run in your terminal: tensorboard --logdir path_to_current_dir/Graph\n",
    "tensorboard_top_layers = TensorBoard(\n",
    "\tlog_dir         = folder + '/logs_top_layers',\n",
    "\thistogram_freq  = 0,\n",
    "\twrite_graph     = True,\n",
    "\twrite_grads     = False,\n",
    "\twrite_images    = True)\n",
    "\n",
    "# Train the model on the new data for a few epochs (Fits the model on data yielded batch-by-batch by a Python generator)\n",
    "    # generator:        A generator or an instance of Sequence (keras.utils.Sequence) object in order to avoid duplicate data when using multiprocessing\n",
    "    #                   The output of the generator must be either {a tuple (inputs, targets)} {a tuple (inputs, targets, sample_weights)}\n",
    "    # steps_per_epoch:  Total number of steps (batches of samples) to yield from generator before declaring one epoch finished and starting the next epoch\n",
    "    #                   It should typically be equal to the number of unique samples of your dataset divided by the batch size \n",
    "    # epochs:           Integer, total number of iterations on the data\n",
    "    # validation_data:  This can be either {a generator for the validation data } {a tuple (inputs, targets)} {a tuple (inputs, targets, sample_weights)}\n",
    "    # callbacks:        List of callbacks to be called during training (to visualize the files created during training, run in your terminal:\n",
    "    #                   tensorboard --logdir path_to_current_dir/Graph)\n",
    "model.fit_generator(\n",
    "    generator           = train_generator,\n",
    "    steps_per_epoch     = len(train_data_x) // batch_size,  # samples_per_epoch / batch_size\n",
    "    epochs              = epochs_top_layers,                            \n",
    "    validation_data     = val_data,\n",
    "    callbacks           = [tensorboard_top_layers])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FULL NETWORK TRAINING ###############################################################################################################################\n",
    "\n",
    "# At this point, the top layers are well trained and we can start fine-tuning convolutional layers from ResNet-50\n",
    "\n",
    "# Fine-tuning of all the layers\n",
    "for layer in model.layers:\n",
    "    layer.trainable = True\n",
    "\n",
    "# We need to recompile the model for these modifications to take effect (we use SGD with nesterov momentum and a low learning rate)\n",
    "    # optimizer:    String (name of optimizer) or optimizer object\n",
    "        # lr:       float >= 0. Learning rate\n",
    "        # momentum: float >= 0. Parameter updates momentum\n",
    "        # decay:    float >= 0. Learning rate decay over each update\n",
    "        # nesterov: boolean. Whether to apply Nesterov momentum\n",
    "    # loss:     String (name of objective function) or objective function\n",
    "    # metrics:  List of metrics to be evaluated by the model during training and testing\n",
    "model.compile(\n",
    "    optimizer   = SGD(lr = 1e-4, momentum = 0.9, decay = 0.0, nesterov = True),\n",
    "    loss        = 'categorical_crossentropy', \n",
    "    metrics     = ['accuracy'])\n",
    "\n",
    "# This callback writes a log for TensorBoard, which allows you to visualize dynamic graphs of your training and test metrics, \n",
    "tensorboard_all_layers = TensorBoard(\n",
    "    log_dir         = folder + '/logs_all_layers',\n",
    "    histogram_freq  = 0,\n",
    "    write_graph     = True,\n",
    "    write_grads     = False,\n",
    "    write_images    = True)\n",
    "\n",
    "def scheduler(epoch):\n",
    "    updated_lr = K.get_value(model.optimizer.lr) * 0.5\n",
    "    if (epoch % 3 == 0) and (epoch != 0):\n",
    "        K.set_value(model.optimizer.lr, updated_lr)\n",
    "        print(K.get_value(model.optimizer.lr))\n",
    "    return K.get_value(model.optimizer.lr)\n",
    "\n",
    "# Learning rate scheduler\n",
    "    # schedule: a function that takes an epoch index as input (integer, indexed from 0) and current learning\n",
    "    #           rate and returns a new learning rate as output (float)\n",
    "reduce_lr = LearningRateScheduler(scheduler)\n",
    "\n",
    "\n",
    "# Reduce learning rate when a metric has stopped improving\n",
    "\t# monitor: \tQuantity to be monitored\n",
    "\t# factor: \tFactor by which the learning rate will be reduced. new_lr = lr * factor\n",
    "\t# patience:\tNumber of epochs with no improvement after which learning rate will be reduced\n",
    "\t# mode: \tOne of {auto, min, max}\n",
    "\t# min_lr:\tLower bound on the learning rate\n",
    "reduce_lr_plateau = ReduceLROnPlateau(\n",
    "\tmonitor \t= 'val_loss',\n",
    "\tfactor\t\t= 0.5,\n",
    "\tpatience\t= 3,\n",
    "\tmode \t\t= 'auto',\n",
    "\tmin_lr\t\t= 1e-8)\n",
    "\n",
    "# Stop training when a monitored quantity has stopped improving\n",
    "\t# monitor:\t\tQuantity to be monitored\n",
    "\t# patience:\t\tNumber of epochs with no improvement after which training will be stopped\n",
    "\t# mode: \t\tOne of {auto, min, max}\n",
    "early_stop = EarlyStopping(\n",
    "\tmonitor \t= 'val_loss',\n",
    "\tpatience \t= 10,\n",
    "\tmode \t\t= 'auto')\n",
    "\n",
    "class ModelCheckpoint(Callback):\n",
    "\n",
    "\tdef __init__(self, filepath, folder, monitor = 'val_loss', verbose = 0, save_best_only = False, save_weights_only = False, mode = 'auto', period = 1):\n",
    "\t\tsuper(ModelCheckpoint, self).__init__()\n",
    "\t\tself.monitor \t\t\t\t= monitor\n",
    "\t\tself.verbose\t\t \t\t= verbose\n",
    "\t\tself.filepath \t\t\t\t= filepath\n",
    "\t\tself.folder \t\t\t\t= folder\n",
    "\t\tself.save_best_only \t\t= save_best_only\n",
    "\t\tself.save_weights_only\t\t= save_weights_only\n",
    "\t\tself.period \t\t\t\t= period\n",
    "\t\tself.epochs_since_last_save\t= 0\n",
    "\t\t\n",
    "\t\tif mode not in ['auto', 'min', 'max']:\n",
    "\t\t\twarnings.warn('ModelCheckpoint mode %s is unknown, ' 'fallback to auto mode.' % (mode), RuntimeWarning)\n",
    "\t\t\tmode = 'auto'\n",
    "\n",
    "\t\tif mode == 'min':\n",
    "\t\t\tself.monitor_op = np.less\n",
    "\t\t\tself.best = np.Inf\n",
    "\t\telif mode == 'max':\n",
    "\t\t\tself.monitor_op = np.greater\n",
    "\t\t\tself.best = -np.Inf\n",
    "\t\telse:\n",
    "\t\t\tif 'acc' in self.monitor or self.monitor.startswith('fmeasure'):\n",
    "\t\t\t    self.monitor_op = np.greater\n",
    "\t\t\t    self.best = -np.Inf\n",
    "\t\t\telse:\n",
    "\t\t\t    self.monitor_op = np.less\n",
    "\t\t\t    self.best = np.Inf\n",
    "\t\n",
    "\tdef on_epoch_end(self, epoch, logs=None):\n",
    "\t\tlogs = logs or {}\n",
    "\t\tself.epochs_since_last_save += 1\n",
    "\t\tif self.epochs_since_last_save >= self.period:\n",
    "\t\t\tself.epochs_since_last_save = 0\n",
    "\t\t\tfilepath = self.filepath.format(epoch = epoch + 1, **logs)\n",
    "\t\t\tif self.save_best_only:\n",
    "\t\t\t\tcurrent = logs.get(self.monitor)\n",
    "\t\t\t\tif current is None:\n",
    "\t\t\t\t    warnings.warn('Can save best model only with %s available, ' 'skipping.' % (self.monitor), RuntimeWarning)\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t\tif self.monitor_op(current, self.best):\n",
    "\t\t\t\t\t    if self.verbose > 0:\n",
    "\t\t\t\t\t        print('\\nEpoch %05d: %s improved from %0.5f to %0.5f,' ' saving model to %s' % (epoch + 1, self.monitor, self.best, current, filepath))\n",
    "\t\t\t\t\t    self.best = current\n",
    "\t\t\t\t\t    if self.save_weights_only:\n",
    "\t\t\t\t\t        self.model.save_weights(filepath, overwrite=True)\n",
    "\t\t\t\t\t    else:\n",
    "\t\t\t\t\t\t\tself.model.save(filepath, overwrite=True)\n",
    "\t\t\t\t\t\t\t# Save model.h5 on to google storage\n",
    "\t\t\t\t\t\t\twith file_io.FileIO(filepath, mode='r') as input_f:\n",
    "\t\t\t\t\t\t\t\twith file_io.FileIO(self.folder + '/checkpoints/' + filepath, mode='w+') as output_f:\t# w+ : writing and reading\n",
    "\t\t\t\t\t\t\t\t\toutput_f.write(input_f.read())\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\tif self.verbose > 0:\n",
    "\t\t\t\t\t\t    print('\\nEpoch %05d: %s did not improve' %\n",
    "\t\t\t\t\t\t          (epoch + 1, self.monitor))\n",
    "\t\t\telse:\n",
    "\t\t\t\tif self.verbose > 0:\n",
    "\t\t\t\t    print('\\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))\n",
    "\t\t\t\tif self.save_weights_only:\n",
    "\t\t\t\t    self.model.save_weights(filepath, overwrite=True)\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t\tself.model.save(filepath, overwrite=True)\n",
    "\t\t\t\t\t# Save model.h5 on to google storage\n",
    "\t\t\t\t\twith file_io.FileIO(filepath, mode='r') as input_f:\n",
    "\t\t\t\t\t\twith file_io.FileIO(self.folder + '/checkpoints/' + filepath, mode='w+') as output_f:\t# w+ : writing and reading\n",
    "\t\t\t\t\t\t\toutput_f.write(input_f.read())\n",
    "\n",
    "# Save the model after every epoch\n",
    "\t# filepath:       String, path to save the model file\n",
    "\t# monitor:        Quantity to monitor {val_loss, val_acc}\n",
    "\t# save_best_only: If save_best_only=True, the latest best model according to the quantity monitored will not be overwritten\n",
    "\t# mode:           One of {auto, min, max}. If save_best_only = True, the decision to overwrite the current save file is made based on either\n",
    "\t#\t\t\t      the maximization or the minimization of the monitored quantity. For val_acc, this should be max, for val_loss this should\n",
    "\t#\t\t\t      be min, etc. In auto mode, the direction is automatically inferred from the name of the monitored quantity\n",
    "\t# period:         Interval (number of epochs) between checkpoints\n",
    "check_point = ModelCheckpoint(\n",
    "\tfilepath\t\t= 'ResNet-50_{epoch:02d}_{val_loss:.2f}.h5',\n",
    "\tfolder \t\t\t= folder,\n",
    "\tmonitor \t\t= 'val_loss', # Accuracy is not always a good indicator because of its yes or no nature\n",
    "\tsave_best_only\t= True,\n",
    "\tmode \t\t\t= 'auto',\n",
    "\tperiod\t\t\t= 1)\n",
    "\n",
    "# We train our model again (this time fine-tuning all the resnet blocks)\n",
    "model.fit_generator(\n",
    "    generator           = train_generator,\n",
    "    steps_per_epoch     = len(train_data_x) // batch_size,  # samples_per_epoch / batch_size \n",
    "    epochs              = epochs_all_layers,                        \n",
    "    validation_data     = val_data,\n",
    "    callbacks           = [tensorboard_all_layers, reduce_lr, reduce_lr_plateau, early_stop, check_point])\n",
    "\n",
    "# SAVING ##############################################################################################################################################\n",
    "\n",
    "# Saving the model in the workspace\n",
    "model.save(folder + '/ResNet-50.h5')\n",
    "# Save model.h5 on to google storage\n",
    "with file_io.FileIO('ResNet-50.h5', mode='r') as input_f:\n",
    "    with file_io.FileIO(folder + '/ResNet-50.h5', mode='w+') as output_f:  # w+ : writing and reading\n",
    "        output_f.write(input_f.read())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
